{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Otto商品分类——RBF 核SVM\n",
    "\n",
    "我们以Kaggle 2015年举办的Otto Group Product Classification Challenge竞赛数据为例，分别调用\n",
    "缺省参数SVC、\n",
    "SVC + GridSearchCV进行参数调优。\n",
    "\n",
    "Otto数据集是著名电商Otto提供的一个多类商品分类问题，类别数=9. 每个样本有93维数值型特征（整数，表示某种事件发生的次数，已经进行过脱敏处理）。 竞赛官网：https://www.kaggle.com/c/otto-group-product-classification-challenge/data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "#竞赛的评价指标为logloss\n",
    "#from sklearn.metrics import log_loss  \n",
    "\n",
    "#SVM虽然也支持输出各类的概率，但这需要额外的计算费用，且得到的概率也不保证是合法的概率，\n",
    "#所以在这个例子中我们用正确率accuracy_score作为模型选择的度量，最后在最佳超参数情况下再训练模型，得到概率表示\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>feat_1_tfidf</th>\n",
       "      <th>feat_2_tfidf</th>\n",
       "      <th>feat_3_tfidf</th>\n",
       "      <th>feat_4_tfidf</th>\n",
       "      <th>feat_5_tfidf</th>\n",
       "      <th>feat_6_tfidf</th>\n",
       "      <th>feat_7_tfidf</th>\n",
       "      <th>feat_8_tfidf</th>\n",
       "      <th>feat_9_tfidf</th>\n",
       "      <th>...</th>\n",
       "      <th>feat_85_tfidf</th>\n",
       "      <th>feat_86_tfidf</th>\n",
       "      <th>feat_87_tfidf</th>\n",
       "      <th>feat_88_tfidf</th>\n",
       "      <th>feat_89_tfidf</th>\n",
       "      <th>feat_90_tfidf</th>\n",
       "      <th>feat_91_tfidf</th>\n",
       "      <th>feat_92_tfidf</th>\n",
       "      <th>feat_93_tfidf</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.081393</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.075886</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.231403</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.199730</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0.011987</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011668</td>\n",
       "      <td>0.105971</td>\n",
       "      <td>0.021681</td>\n",
       "      <td>0.080435</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.008244</td>\n",
       "      <td>0.022456</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.124622</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.145988</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 95 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  feat_1_tfidf  feat_2_tfidf  feat_3_tfidf  feat_4_tfidf  feat_5_tfidf  \\\n",
       "0   1      0.081393           0.0           0.0      0.000000      0.000000   \n",
       "1   2      0.000000           0.0           0.0      0.000000      0.000000   \n",
       "2   3      0.000000           0.0           0.0      0.000000      0.000000   \n",
       "3   4      0.011987           0.0           0.0      0.011668      0.105971   \n",
       "4   5      0.000000           0.0           0.0      0.000000      0.000000   \n",
       "\n",
       "   feat_6_tfidf  feat_7_tfidf  feat_8_tfidf  feat_9_tfidf  ...  feat_85_tfidf  \\\n",
       "0      0.000000      0.000000      0.000000           0.0  ...       0.075886   \n",
       "1      0.000000      0.000000      0.231403           0.0  ...       0.000000   \n",
       "2      0.000000      0.000000      0.199730           0.0  ...       0.000000   \n",
       "3      0.021681      0.080435      0.000000           0.0  ...       0.000000   \n",
       "4      0.000000      0.000000      0.000000           0.0  ...       0.124622   \n",
       "\n",
       "   feat_86_tfidf  feat_87_tfidf  feat_88_tfidf  feat_89_tfidf  feat_90_tfidf  \\\n",
       "0       0.000000       0.000000            0.0            0.0       0.000000   \n",
       "1       0.000000       0.000000            0.0            0.0       0.000000   \n",
       "2       0.000000       0.000000            0.0            0.0       0.000000   \n",
       "3       0.008244       0.022456            0.0            0.0       0.000000   \n",
       "4       0.000000       0.000000            0.0            0.0       0.145988   \n",
       "\n",
       "   feat_91_tfidf  feat_92_tfidf  feat_93_tfidf   target  \n",
       "0            0.0            0.0            0.0  Class_1  \n",
       "1            0.0            0.0            0.0  Class_1  \n",
       "2            0.0            0.0            0.0  Class_1  \n",
       "3            0.0            0.0            0.0  Class_1  \n",
       "4            0.0            0.0            0.0  Class_1  \n",
       "\n",
       "[5 rows x 95 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取数据\n",
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "\n",
    "#原始特征 + tf_idf特征对线性SVM训练还是很快，RBF核已慢得不行\n",
    "# RBF核只用tf_idf特征\n",
    "train = pd.read_csv(dpath +\"Otto_FE_train_tfidf.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将类别字符串变成数字\n",
    "# drop ids and get labels\n",
    "y_train = train['target']   #形式为Class_x\n",
    "X_train = train.drop([\"id\", \"target\"], axis=1)\n",
    "\n",
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns \n",
    "\n",
    "#sklearn的学习器大多之一稀疏数据输入，模型训练会快很多\n",
    "from scipy.sparse import csr_matrix\n",
    "X_train = csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 训练样本6w+，交叉验证太慢，用train_test_split估计模型性能\n",
    "# SVM对大样本数据集支持不太好\n",
    "# 随机抽取10000条记录作为训练样本\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train_part, X_val, y_train_part, y_val = train_test_split(X_train, y_train, train_size = 10000,random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000, 93)\n"
     ]
    }
   ],
   "source": [
    "print (X_train_part.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RBF核SVM正则参数调优\n",
    "\n",
    "RBF核是SVM最常用的核函数。\n",
    "RBF核SVM 的需要调整正则超参数包括C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和核函数的宽度gamma\n",
    "C越小，决策边界越平滑； \n",
    "gamma越小，决策边界越平滑。\n",
    "\n",
    "采用交叉验证，网格搜索步骤与Logistic回归正则参数处理类似，在此略。\n",
    "\n",
    "这里我们用校验集（X_val、y_val）来估计模型性能"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_grid_point_RBF(C, gamma, X_train, y_train, X_val, y_val):\n",
    "    \n",
    "    # 在训练集是那个利用SVC训练\n",
    "    SVC3 =  SVC( C = C, kernel='rbf', gamma = gamma)\n",
    "    SVC3 = SVC3.fit(X_train, y_train)\n",
    "    \n",
    "    # 在校验集上返回accuracy\n",
    "    accuracy = SVC3.score(X_val, y_val)\n",
    "    \n",
    "    print(\"C= {} and gamma = {}: accuracy= {} \" .format(C, gamma, accuracy))\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy_s = np.matrix(np.zeros(shape=(5, 3)), float)\n",
    "gamma_s = np.logspace(-1, 1, 3)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C= 0.1 and gamma = 0.1: accuracy= 0.7094722232931108 \n",
      "C= 0.1 and gamma = 1.0: accuracy= 0.7350707428967963 \n",
      "C= 0.1 and gamma = 10.0: accuracy= 0.4446007941709395 \n"
     ]
    }
   ],
   "source": [
    "oneC = 0.1\n",
    "\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    accuracy_s[0,j] = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C= 1 and gamma = 0.1: accuracy= 0.7457689193877944 \n",
      "C= 1 and gamma = 1.0: accuracy= 0.7763020933729133 \n",
      "C= 1 and gamma = 10.0: accuracy= 0.643451944947762 \n"
     ]
    }
   ],
   "source": [
    "oneC = 1\n",
    "\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    accuracy_s[1,j] = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C= 10 and gamma = 0.1: accuracy= 0.7672231003508231 \n",
      "C= 10 and gamma = 1.0: accuracy= 0.789544701029338 \n",
      "C= 10 and gamma = 10.0: accuracy= 0.6561548247812174 \n"
     ]
    }
   ],
   "source": [
    "oneC = 10\n",
    "\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    accuracy_s[2,j] = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C= 100 and gamma = 0.1: accuracy= 0.781371679710089 \n",
      "C= 100 and gamma = 1.0: accuracy= 0.7751262577585875 \n",
      "C= 100 and gamma = 10.0: accuracy= 0.6549789891668916 \n"
     ]
    }
   ],
   "source": [
    "oneC = 100\n",
    "\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    accuracy_s[3,j] = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C= 1000 and gamma = 0.1: accuracy= 0.7719457188018042 \n",
      "C= 1000 and gamma = 1.0: accuracy= 0.7672423763445005 \n",
      "C= 1000 and gamma = 10.0: accuracy= 0.6549789891668916 \n"
     ]
    }
   ],
   "source": [
    "oneC = 1000\n",
    "\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    accuracy_s[4,j] = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#需要调优的参数\n",
    "#C_s = np.logspace(-1, 3, 5)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份 \n",
    "#gamma_s = np.logspace(-1, 1, 3)    \n",
    "\n",
    "#accuracy_s = []\n",
    "#for i, oneC in enumerate(C_s):\n",
    "    #for j, gamma in enumerate(gamma_s):\n",
    "        #tmp = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)\n",
    "        #accuracy_s[i,j] = tmp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从上述结果会发现，gamma参数非常重要(当gamma=0.01或gamma=100时性能很差),非线性模型比线性模型性能好（注意我们这里只用了tfidf特征）。\n",
    "但速度慢了不是一点半点(sklearn建议核方法SVM样本数不超过10000)\n",
    "可以考虑将训练样本分为多个子集，每个子集训练一个RBF核SVM模型，最后多个模型融合的结果的到最终模型（训练速度加快，但测试可能更慢）\n",
    "\n",
    "C= 0.1 and gamma = 0.1: accuracy= 0.738122171946 \n",
    "C= 0.1 and gamma = 1.0: accuracy= 0.763897866839 \n",
    "C= 0.1 and gamma = 10.0: accuracy= 0.549208144796 \n",
    "\n",
    "C= 1 and gamma = 0.1: accuracy= 0.761716224952 \n",
    "C= 1 and gamma = 1.0: accuracy= 0.800985778927 \n",
    "C= 1 and gamma = 10.0: accuracy= 0.71630575307\n",
    "\n",
    "C= 10 and gamma = 0.1: accuracy= 0.78603749192 \n",
    "C= 10 and gamma = 1.0: accuracy= 0.817065287654 \n",
    "C= 10 and gamma = 10.0: accuracy= 0.727052359405 \n",
    "\n",
    "C= 100 and gamma = 0.1: accuracy= 0.805914673562 \n",
    "#### C= 100 and gamma = 1.0: accuracy= 0.807449903038 \n",
    "C= 100 and gamma = 10.0: accuracy= 0.724951519069\n",
    "\n",
    "C= 1000 and gamma = 0.1: accuracy= 0.805914673562 \n",
    "C= 1000 and gamma = 1.0: accuracy= 0.794521654816 \n",
    "C= 1000 and gamma = 10.0: accuracy= 0.724062702004 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXiU5dX48e/Jvu8LJCEEBFkSWcNS0Kq4IYKopUVbX/ftdalL609tebsgKNpS5VW0KlLXiq1apdRWcatvVSBhh6CyCCEECJCQhITs9++P58lkkkxgEmYyWc7nuubKzLNMzgxkzjz3cm4xxqCUUkq15OfrAJRSSnVNmiCUUkq5pAlCKaWUS5oglFJKuaQJQimllEsBvg7AUxISEkxGRoavw1BKqW5l7dq1h40xia729ZgEkZGRQW5urq/DUEqpbkVE9rS1T5uYlFJKuaQJQimllEuaIJRSSrmkCUIppZRLmiCUUkq5pAlCKaWUS5oglFJKudRj5kEo1aXU10LJHjj8LRzZDghEJENEkv0zGUJjwU+/o6muSxOEUqfi+FE4ssNKBIe/hcPbrZ/Fu6Ch7sTn+gVAeBJEJLZOHs73wxMhOBJEOuc1KWXTBKHUyTQ0QOle60qgMQE0/jx2sOk4vwCIOw0SToeh062fCadDwiBA4FiRdfyxg073nbYd2Gw9NvWtYwgIdUoaLROJ0/bwJAgM6bS3RvVsmiCUalRT6XQ14JQIjuyAuuNNx4XEWB/8gy6AhMFNiSC2P/gHtv38IVF2sjiBhgY4XtxGIrHvH9kJe760jnP5e6KbJ5DwJNdXJ+EJ4Off/vdJ9RqaIFTvYoz1IetoEnJKCKX5TgeK9YGfcDoMPLt5IgiL915zj5+f9cEdngDJmSc+tq4GKg5Zr6fxZ8ukUrje+llzrPX54gdhCa2vQlolkkSrv0SbuHodTRCqZ6qrgZLvWvcNHN4O1WVNxwWGW9/q0ydAwn81JYK407p+U01AEESnWreTqalofhXSMpFUFFnvzbGDUF/T+nz/oBZXIkmum7gikiEo3POvVfmEJgjVvVUWO334f9vURFT8XfO2/MgU68N/xGz7SsBOBFEpveObcVA4xA2wbidiDFQdddFH4pRcSgtg31rrqgXj4ndFtO4XaZVU7CuTgCCvvFzlGZogVNfXUA9H85sngsb7lYebjvMPgvhBkDQchl/mlAgGW6OA1MmJWM1JobGQOOTEx9bXQeURu4mryHVSKdoGxz6DqlLXzxEa1+IqxE4gUakQk27dwpN0OLCPaIJQXUf1sRYjhRo7iXdCfXXTcWHx9kihaU4jhQZDTH/tdO1M/gEQmWzdTqa2yu4ncdXEZfehFORgyg8izgMCgAa/IKrCU61bRBpV4fYtIpWq8DRqQxJ6x1XgCUSGBDAmPdbjz6sJQnUuY6Cs0HUiKNvXdJz4QewAe7TQeU2JIH4whMf7Ln7VMYEhENPPutnKqmrJKyxjy75S8grL2FpWxo7KckIbKukjxaTJIdLkMGlyiH61RaSVFpImG0mR8mZPXWUC2WcSKDCJTrcE9pokCkwih4kCenYCGdUvhnfvmOzx59UEobyjtsqaLNasg9juI3AeURMUaX37zzir+UihuAEQEOy7+JVHHSqvZkuhnQgKS9myr4z84krH/qTIYDJTorgwM5nMlCgSI1sPEKgDdgP5tRUEHdtH0LECgo8VEHSsgLhjBfQp38vkY+sIqC5pdl6DfzDVEWnU2LfG+9WR1s+6HnAFEh7snStnryYIEZkKLAL8gSXGmAUt9qcDLwMx9jEPGmPet/c9BNwI1AM/NcZ84M1YVQcYY7VBtxwldPhbOLoHTEPTsdH9rASQfrXVT9CYCCL7dPs/TtXEGENByXG2FpaytbCMrfYVQlF5UxNhelwYWalRzB7Xj+EpUWSmRJHkIiG0LRZIAya43l1dDkf3Wv1WR/PxO7qH0KP5hB7Nh/zNreePBIQ29Xc0u/W3+0C6fwLpKK8lCBHxBxYDFwAFQI6ILDfG5DkdNgf4izHmWREZDrwPZNj3rwQygRTgIxE53RhXU0xVp2logL2rYMs7cGCTlQiOO31bCwixPvxTRsGIHzX1DcQP0qGPPVB9g2HXoWN2IrCuCvL2l1F6vBYAP4HBSZGcOSiB4SlRZKVGMzwliqiQE0wm9ITgSEgebt1cqS53JI+m2x7r577c5v+noSmBxPZ3nUS8OS/Gx7x5BTEe2GGM2QUgIsuAmYBzgjBAlH0/Gii0788ElhljqoHvRGSH/XxfeTFe1ZZD38CmN2HTX63JZIFhkDKm9Uih6H7aSdxDVdfV8+2BY1YisK8Ovt5fzvFa6ztbUIAfw/pEMu2MvmTayWBon0hCArvg/4fgSGsSYlsTEavKrNIqR/OtgovOCWTvGmsYsLPAsNZXHc0SSFy3TSDeTBCpwF6nxwW0vib8DfChiNwFhAPnO527qsW5rWYDicgtwC0A6enpHgla2coPwJa3rcSwf6PVaTzwXJgyB4ZeAsERvo5Qecmx6jq27beahhqbibYfLKeuwZrzEBEcwPCUKK4c34+slGgyU6M4LTGCQP8eMhQ1JApCTpRASps1YTUlkD2wd3XrIb2B4a2vPBxXI/279Cx1byYIV6+45ayaq4CXjDELReR7wKsikuXmuRhjngeeB8jOznYxY0e1S/Ux+HqFlRR2fWb1IfQdBRc9Clk/cG84o+pWiitqHM1DW+1O5O+OVGDsv6b48CAyU6M5Z0iilQxSokiPC8PPr2t+oHWKkGjoEw19slzvP3606QqkZRLJXwXVLRJIUEQbfSC+TyDeTBAFQD+nx2k0NSE1uhGYCmCM+UpEQoAEN89VnlBfZyWDTW9ayaG2EqLT4cz7rH6Ek02WUt2CMYb9pVXNrgq2Fpayv7TKcUxqTCiZKVFcNjqVzJQoMlOiSY4KRrrot9suKzTGuvU5w/X+40ddJA/7tufL5qVgwBrp12YCSfdqAvFmgsgBBovIAGAfVqfzj1sckw+cB7wkIsOAEOAQsBz4s4j8AauTejCwxoux9i7GWEXcNv0FtrxlTVIKibHKUIyYDf0m6MzVbqyhwfDdkQpHEthqXx2UVFqdxyIwMCGc8QPiHIkgMyWKmDAte9EpGhNI3xGu9x8/2tTn0fK2+z9Q03weCMFRMOD7cOXrHg/VawnCGFMnIncCH2ANYV1qjNkqInOBXGPMcuBnwAsici9WE9J1xhgDbBWRv2B1aNcBd+gIJg8o2W11NG9605qo5h8Ep0+1ksLgC3TeQTdUU9fA9qJyKxnYVwfb9pdRUWP9uQT6C6cnR3Lh8D5kplrJYFjfSMKCdApUl+VIICNb72usleWcNEr2WB3hXiDG9Iym++zsbJObm+vrMLqeymLY+jframGv3e/ff7LVfDR8pnV5qrqFypo6tu0vJ6+xz2B/Kd8eOEZNvTXfJCzIn+F9o5quClKjGJwUSVCAXg2qtonIWmNMtqt9+jWiJ6qtgu0fwMY3YfuH0FALCUPgvF/BGT+02i1Vl1ZaWdtsSOnWwjJ2HTqGPZCI2LBAMlOiuX5yBpmpVhNRRnw4/r2581h5nCaInqKhAfK/tJqPtr5njZSISIYJt1pXC31GdNmhdL2ZMYai8mqnzmPr6mDf0aaCdX2jQ8hMieISe45BZmo0KdEh2nmsvE4TRHdXtK1pEltZgTXmetgMGDkbBpytE9e6kIYGQ35xZVMiKCwjr7CUw8eaFugZkBDOqPQYrp7Y324qiiI+QvuGlG9oguiOyvZbo482vWktdC/+cNoUOP83VglsLWvRJRw5Vk3O7hJydxezaV8p2wrLKK+uAyDATxiUFME5Q5IcfQbD+kYS6e0yFEq1gyaI7qK6HLbZk9i++7c1iS1lDEx9DLKusBZZUT5jjHV1kLO7hJzvisnZU8yuQxWAVYZieN8oZo5OITMlmqyUaAYnR3TNMhRKOdEE0ZXV18LOT+1JbP+AuuPWzMqzfm4Xwxvs6wh7rfoGw7b9ZeTsLiZ3dwk5u4sdFUujQgIYlxHHD8f2Y1xGLGekRRMcoMlAdT+aILoaY2DfOispbHnbWlIzJAZGXQUjroR+47Wz2QeqautZn3+U3N3FrNldzPr8oxyzm4tSY0L53mnxZGfEMT4jjsFJEb27FIXqMTRBdBXFu5omsRXvBP9gGGJPYht0gS7u3slKKmrI3VPiSAhb9pVSW2+NMR2SHMllo1MYlxFHdkYcqTGhPo5WKe/QBOFLlcV2xdS/QIFdSSTjLDjzHhh2qTWbUnld4yI3ObuLHZ3K24usVe+C/P0YkRbNjWcOZFxGLGP7x2pJCtVraILobLXH4dt/WUlh+4fQUAeJw6wRSFmzmq3Zq7yjvsHwzYFycvcUs+Y7qw/hQJlVtC4yOICxGbFcNjqVcRlxjEiL1s5k1WtpgugMDQ2w5z9W81HecqtaY2RfmPjfVhNScpb2K3hRVW09mwpK7SuEYtbuKaG8yuo/6BMVwrgBcYzLiCW7fxxD+kTqbGSlbJogvOngVispbH4LyvZZdd+HXWqNQBrwfZ3E5iWllbWszS9mzXf2HISCUke9osFJEUwfkcK4jFjGZcSRFhuqM5KVaoMmCE8rK4TNf7WakA5usSaxDTofLpgLQ6ZBUJivI+xxCo8ed1wd5HxXwjcHrXLIAX7CGWnRXDc5g3EZcYztH0tcuPYfKOUuTRCeUFUG2/5uT2L7HDCQmg0X/w4yL4eIRF9H2GM0NBi2Fx1zJITc3SWOukURwQGM6R/L9BF9yc6IY1S/GEKD9CpNqY7SBNFR9bWw42MrKXzzPtRVQewAOPsBqwkp/jRfR9gjVNfVs2VfqaO5KHdPCaXHrYVvEiODGZ8Rx01nDWBcRhxD+0QS0FPWRVaqC9AE0R7GQEGuXTH1Hag8AqFxMPpqq7M5bZx2Np+isqpa1trzD3K+K2FDwVFq6qz+g4GJ4UzN7OPoVE6PC9P+A6W8SBOEO47stPsV3rQmtAWEwJCLraRw2nk6ie0UHCitauo/2F3C1wfKMAb8/YSs1Giumdif7Iw4sjNiSdCqpkp1Kk0Qbak4bK/E9iYU5AACA86Cs35mldMOifZ1hN2OMYadh445moty9hSzt9jqPwgL8mdMeix3nzeY8RlxjEqP0WUxlfIx/Qt0Vnvc6k/Y9BfY8ZE1iS0p0xqBlDULolN9HWG3UlPXwJbCUisZ2DOUSyqt/oOEiCCy+8dx3aQBjMuIZXjfKO0/UKqL0QTRUA+7/89KCnnLoaYcIlNg4u1WE1KfLF9H2G0cq65jnVP9og17j1JVa/UfDEgI5/xhyYzLiGPcgDgy4rX/QKmuThNE6V54ZSYERcLwmdYIpIwzdRKbG4rKqqz1D3YXk7unmLzCMhoM+AlkpkRz1fh0xmfEMTYjlqTIEF+Hq5RqJ00QsRlw9TvQfxIEalXOtjQ0WP0HVoXTEnL3FLPnSCUAIYF+jO4Xy51TBjMuI5bR6bFEBOt/LaW6O/0rBhh0nq8j6HIqa+rYuLeUtXus2kXr8o865h/EhQeR3T+Wqyf0Z9yAODJTogjU/gOlehxNEAqwhpvm2slg7Z4S8grLqGuw1j8YnBTBxVl9GNs/luwM7T9QqrfQBNEL1dU38PWBckcyWLunqVxFSKAfo/rFcOvZA8nuH8fo9Bhd/0CpXkoTRC9QVlXL+vyjdjIoZkP+USpq6gFIjgomu38cN545gLH9YxmuzUVKKZsmiB7GGMPe4uOszbcK2a3dY1U3NfbooqF9ovjB2DTG9rdWR0uN0XLXSinXNEF0czV1DWwtLLXrF5WwNr+EQ+XVgFXddHR6DBdn9WVs/1hGpcfo6CKllNu8+mkhIlOBRYA/sMQYs6DF/ieAc+2HYUCSMSbG3lcPbLb35RtjLvVmrN1FcUWNNRltTwnr9pSwseAo1XYxu35xoZw5KIEx/WPJ7h/L6cm6OppSquO8liBExB9YDFwAFAA5IrLcGJPXeIwx5l6n4+8CRjs9xXFjzChvxdcdWLWLKhxDTXP3lLDrUAUAgf5CZko0V0/sT7bdXJQUpZPRlFKe480riPHADmPMLgARWQbMBPLaOP4q4NdejKfLq6qtZ+Peo6zNL2Gt3Vx01K5dFBsWyNj+scwam0Z2/zhGpEUTEqizvZVS3uPNBJEK7HV6XABMcHWgiPQHBgCfOG0OEZFcoA5YYIx518V5twC3AKSnp3so7M5TVFZFrj3MNHdPCVv3lTrmHpyWGM6Fw5PJ7h/HmP6xnJYYrp3JSqlO5c0E4erTzLRx7JXAW8aYeqdt6caYQhEZCHwiIpuNMTubPZkxzwPPA2RnZ7f13F1CfYPhmwPl9tVBMWvzSxylroMD/BiZFsPN3x9Idn+rVIWunayU8jVvJogCoJ/T4zSgsI1jrwTucN5gjCm0f+4Skc+w+id2tj61azpWXcf6/KaJaOvzj3Ksug6wlsrM7h/Ltd/LYGz/WDJTogkK0LkHSqmuxZsJIgcYLCIDgH1YSeDHLQ8SkSFALPCV07ZYoNIYUy0iCcBk4HEvxnpKjDHsO3q8aajpHmtltAZjrUA6JDmSy0anWKUq+seRFqtzD5RSXZ/XEoQxpk5E7gQ+wBrmutQYs1VE5gK5xpjl9qFXAcuMMc5NRMOA50SkAfDD6oNoq3O709XWN5BXWOYYapq7p5iDZdbcg/Agf0anx3LXlMGM7R/L6PQYIkMCfRyxUkq1nzT/XO6+srOzTW5urlee+2hlDevs5qLc3dbcg8aFcFJjQsnOiHXMTB6SHKkroymlug0RWWuMyXa1T6fVtmCM4bvDFU5XByXsKDoGQICfkJkSxVXj08nuH8fY/rH0ida5B0qpnqnXJ4iaugY2Fhx1XB2syy+huKIGgOjQQMakx3D56FTG9o9lZFoMoUE690Ap1Tv0+gRRXFHDD/9o9Y8PTAhnytAkx8zk0xIj8NNSFUqpXqrXJ4g+0SH86fpxjEiNJj4i2NfhKKVUl9HrEwTAuUOSfB2CUkp1OTrcRimllEuaIJRSSrmkCUIppZRLmiCUUkq5pJ3USnlZZW0ltQ21BPoFEuAXQIBfAH6i381U16cJQqkOqm2o5cjxIxRVFlFUWcTByoMcqjzkeFx03PpZUVvR6lw/8WuWMALE+um8rfF+s+NaPhbXx7Y8x+XzSCCB/oGO5zjZeS1/apLr+TRBKNWCMYaymrKmD/qWN/uD/8jxI5gWS5wE+AWQGJpIUlgSg2IGMSllEomhiQT7B1PXUEdtQ23TT1Nn3a9vuu98jPOttqGWqroqx3m19fYxbZxX32xpFe/wE7+mBNVGonGV8JolNhfnBfoF4u/njyD4iR9+4oeI4C/+1n2n7Y797m4TwQ8/j57fuM1f/K37Tue39Tra2tbVqjxrglC9SnV9NUWVRY5v+s7f+g9WHuTQcet+dX11q3NjgmNICksiKSyJYXHDSAyzEkFyWLIjKcSGxHaJb9YNpsFl4miWfExd66R1ggTlfF5tfW2zJOfqPFePnZOcq+TYeFy9qccYQ4NpaJWEe7LGxOEqIZ5o27C4YSyassjj8WiCUD1Cg2mguKrY8eF/sPKgdf+40/3KQxytPtrq3GD/YMcHf1ZCFkmh1v2k8CTH/cQw6yqgu/ATP4L8gwjy7/4rEzYmigYamu433lxts5NKy20dOb9lomq5rfH8ZtvsY11ua6hvFlvjtlN9bWmRaV557zVBqC6vsrbS8U3f+YPf+Qrg0PFD1DXUNTtPEBJCE0gMSyQ1IpUxSWMc3/STw5IdVwBRQVFd7tJeNWn8luyPFsrsbJoglM/UNdRx+Pjhk37rP1Z7rNW5EYERjm/24/qMs+6HJpIcluzYnhCaQICf/hdXqqP0r0d53Ak7eY833XfZySsBJIYlkhiW2KyT1/lbf3JYMmGBYT56dUr1HpogVLuVVpey8+hO68O+ovUH/6HKQ1TVV7U6r7GTNzEskaFxQ11+648LiesSnbxKKU0Qqp0OVBzgB8t/QFlNmWNbYydvYmgiWfFZJPVLcnT6Nt66WyevUkoThGqnBWsWUFNfw6JzF9Evsp928irVg2mCUG77bO9nfJz/MfeMuYcp6VN8HY5Sysu0sVe5pbK2kkdXP8qgmEFck3mNr8NRSnUCvYJQbnlu03MUVhTy0tSXCPQL9HU4SqlO4NYVhIi8LSKXiOjwkt5oe8l2Xtn6CpcPupyxyWN9HY5SqpO4+4H/LPBjYLuILBCRoV6MSXUhDaaBeavmEREUwb1j7/V1OEqpTuRWgjDGfGSM+QkwBtgNrBSRL0XkehHR9oYe7L0d77GuaB33jb2P2JBYX4ejlOpEbjcZiUg8cB1wE7AeWISVMFZ6JTLlcyVVJSxcu5AxSWOYOWimr8NRSnUyd/sg3gH+DwgDZhhjLjXGvGmMuQuIOMF5U0XkGxHZISIPutj/hIhssG/fishRp33Xish2+3Zt+1+aOlVPrH2CipoK5kyco7ObleqF3B3F9LQx5hNXO4wx2a62i4g/sBi4ACgAckRkuTEmz+nce52OvwsYbd+PA34NZAMGWGufW+JmvOoUrT24lr/t+Bs3ZN3A4NjBvg5HKeUD7n4tHCYiMY0PRCRWRG4/yTnjgR3GmF3GmBpgGXCidoqrgDfs+xcBK40xxXZSWAlMdTNWdYpq62uZt2oeKeEp3DriVl+Ho5TyEXcTxM3GGEfzj/2hffNJzkkF9jo9LrC3tSIi/YEBQONVitvnKs97Je8VdhzdwS8m/EKrpirVi7mbIPzEqdiO3Xx0sqWqXBXnaWvtwCuBt4xxLKTr1rkicouI5IpI7qFDh04SjnLHvmP7+OPGP3Je+nmc3e9sX4ejlPIhdxPEB8BfROQ8EZmC1RT0r5OcUwD0c3qcBhS2ceyVNDUvuX2uMeZ5Y0y2MSY7MTHxJOGokzHG8MjqRxARHhzfakyBUqqXcTdBPIDV/PPfwB3Ax8D/O8k5OcBgERkgIkFYSWB5y4NEZAgQC3zltPkD4EK7ryMWuNDeprzok/xP+Lzgc+4YdQd9wvv4OhyllI+5NYrJGNOANZv6WXef2BhTJyJ3Yn2w+wNLjTFbRWQukGuMaUwWVwHLjDHG6dxiEXkYK8kAzDXGFLv7u1X7VdRW8OiaRxkSO4SfDPuJr8NRSnUBbiUIERkMPAoMB0IatxtjBp7oPGPM+8D7Lbb9qsXj37Rx7lJgqTvxqVP3zIZnKKosYuE5C3UdZ6UU4H4T05+wrh7qgHOBV4BXvRWU6lxfF3/N69teZ9bpsxiZONLX4Silugh3E0SoMeZjQIwxe+xv/bpiTA/QYBp4+KuHiQ6O5u4xd/s6HKVUF+JuW0KVXep7u92vsA9I8l5YqrO89e1bbDq8iUfOfITo4Ghfh6OU6kLcvYK4B6sO00+BscDVgNZH6uYOHz/Mk+ueZHyf8UwfON3X4SilupiTXkHYk+J+ZIy5HzgGXO/1qFSnWJi7kKq6KuZMnIPTPEillALcuIKwZzePFf0E6VFW71/Nil0ruCHrBgZED/B1OEqpLsjdPoj1wHsi8legonGjMeYdr0SlvKqmvoZ5q+bRL7IfN51xk6/DUUp1Ue4miDjgCM1HLhlAE0Q3tHTLUnaX7eaP5/+RkICQk5+glOqV3J1Jrf0OPUR+WT4vbHqBqRlTmZw62dfhKKW6MHdnUv8JF9VUjTE3eDwi5TXGGOavnk+QfxD3j7vf1+Eopbo4d5uYVjjdDwEup+3KrKqL+mD3B3xZ+CUPjX+IpDCdxqKUOjF3m5jedn4sIm8AH3klIuUV5TXlPJbzGMPjhzN7yGxfh6OU6gY6WpVtMJDuyUCUdz21/imOHD/C01Oext/P39fhKKW6AXf7IMpp3gdxAGuNCNUNbD28lWVfL+PKoVeSmZDp63CUUt2Eu01Mkd4ORHlHfUM9v/3qt8SHxnPX6Lt8HY5SqhtxqxaTiFwuItFOj2NE5DLvhaU8Zdk3y9hWvI0Hxj1AZJDmeaWU+9wt1vdrY0xp4wNjzFHg194JSXlKUWURT61/ikkpk7go4yJfh6OU6mbcTRCujtNlx7q43+X8jtr6Wn454ZdajE8p1W7uJohcEfmDiJwmIgNF5AlgrTcDU6fmi31f8K/d/+LmETeTHqUDzpRS7edugrgLqAHeBP4CHAfu8FZQ6tRU1VUxf/V8MqIyuCFLJ7srpTrG3VFMFcCDXo5FeciSzUvYW76XJRcuIcg/yNfhKKW6KXdHMa0UkRinx7Ei8oH3wlIdtat0Fy9ueZHpA6czoe8EX4ejlOrG3G1iSrBHLgFgjClB16TucowxzF81n9CAUH6W/TNfh6OU6ubcTRANIuLo6RSRDFxUd1W+tWLXCtYcWMM9Y+4hITTB1+Eopbo5d4eq/hL4j4j82378feAW74SkOqK0upTf5/6eEYkjmHX6LF+Ho5TqAdztpP6XiGRjJYUNwHtYI5lUF7Fo3SJKq0t5/oLn8RN3LwyVUqpt7hbruwm4G0jDShATga9ovgSp8pENRRv467d/5Zrh1zAkboivw1FK9RDuftW8GxgH7DHGnAuMBg55LSrltrqGOh5e9TDJYcncPup2X4ejlOpB3E0QVcaYKgARCTbGfA2c9KuqiEwVkW9EZIeIuJxHISI/EpE8EdkqIn922l4vIhvs23I34+x1Xt/2Ot+WfMtD4x8iPDDc1+EopXoQdzupC+x5EO8CK0WkhJMsOSoi/sBi4AKgAMgRkeXGmDynYwYDDwGTjTElIuI8dPa4MWZUO15Lr3Og4gCLNyzm7LSzmZKurX1KKc9yt5P6cvvub0TkUyAa+NdJThsP7DDG7AIQkWXATCDP6ZibgcX2vAqMMUXtiL3XW7BmAcYYHprwkBbjU0p5XLuHuxhj/m2MWW6MqTnJoanAXqfHBfY2Z6cDp4vIFyKySkSmOu0LEZFce7uuPdHCZ3s/4+P8j7lt5G2kRrR8W5VS6tR5s2S3q6+0LSfXBWCtb30O1gip/xORLHvWdroxplBEBgKfiMhmY8zOZr9A5Bbs+Rjp6b2nYmllbSWPrn6UQTGDuCbzGl+Ho5Tqobw5YL4A6Of0OI3W/RYFwHvGmFpjzHfAN1gJA2NMof1zF/AZ1sipZowxzxtjso0x2XIjhKMAAB12SURBVImJiZ5/BV3Uc5ueo7CikDkT5xDoF+jrcJRSPZQ3E0QOMFhEBohIEHAl0HI00rvAuQAikoDV5LTLLgYY7LR9Ms37Lnqt7SXbeWXrK1w+6HLGJo/1dThKqR7Ma01Mxpg6EbkT+ADwB5YaY7aKyFwg1xiz3N53oYjkAfXA/caYIyIyCXhORBqwktgC59FPvVWDaWDeqnlEBEVw79h7fR2OUqqH8+qyocaY94H3W2z7ldN9A9xn35yP+RI4w5uxdUfv7XiPdUXrmDtpLrEhsb4ORynVw2nRnm6ipKqEhWsXMiZpDDMHzfR1OEqpXkATRDfxh7V/oKKmgjkT52gxPqVUp9BPmm4g90Au7+54l2syr2Fw7GBfh6OU6iU0QXRxtfW1zFs1j5TwFG4dcauvw1FK9SJe7aRWp+7lvJfZWbqTp6c8TVhgmK/DUUr1InoF0YUVlBfw3MbnOC/9PM7ud7avw1FK9TKaILooYwyPrnkUEeHB8S4rpSullFdpguiiPsn/hM8LPueOUXfQJ7yPr8NRSvVCmiC6oIraCh5d8yhDYofwk2E/8XU4SqleSjupu6BnNjxDUWURC89ZSICf/hMppXxDryC6mK+Lv+b1ba8z6/RZjEwc6etwlFK9mCaILqTBNPDwVw8THRzN3WPu9nU4SqleThNEF/LWt2+x6fAmfp79c6KDo30djlKql9ME0UUcPn6YJ9c9yfg+45k+cLqvw1FKKU0QXcXC3IVU1VUxZ+IcRFyt1qqUUp1LE0QXsHr/albsWsENWTcwIHqAr8NRSilAE4TP1dTXMG/VPPpF9uOmM27ydThKKeWgg+x9bOmWpewu280fz/8jIQEhvg5HKaUc9ArCh/LL8nlh0wtMzZjK5NTJvg5HKaWa0QThI8YY5q+eT5B/EPePu9/X4SilVCuaIHzkg90f8GXhl9w1+i6SwpJ8HY5SSrWiCcIHymvKeSznMYbHD2f2kNm+DkcppVzSTmofeGr9UxRXFfP0eU/j7+fv63CUUsolvYLoZFsPb2XZ18uYPWQ2mfGZvg5HKaXapAmiE9U31PPbr35LfGg8d42+y9fhKKXUCWmC6ETLvlnGtuJtPDDuASKDIn0djlJKnZD2QXSSosoinlr/FJNSJnFRxkW+Dkd1cbW1tRQUFFBVVeXrUFQPERISQlpaGoGBgW6fowmikzye8zi19bX8csIvtRifOqmCggIiIyPJyMjQ/y/qlBljOHLkCAUFBQwY4H69N682MYnIVBH5RkR2iMiDbRzzIxHJE5GtIvJnp+3Xish2+3atN+P0tv/s+w8f7P6Am0fcTHpUuq/DUd1AVVUV8fHxmhyUR4gI8fHx7b4i9doVhIj4A4uBC4ACIEdElhtj8pyOGQw8BEw2xpSISJK9PQ74NZANGGCtfW6Jt+L1lqq6Kuavmk9GVAY3ZN3g63BUN6LJQXlSR/4/efMKYjywwxizyxhTAywDZrY45mZgceMHvzGmyN5+EbDSGFNs71sJTPVirF7zwuYXKDhWwJyJcwjyD/J1OEop5TZvJohUYK/T4wJ7m7PTgdNF5AsRWSUiU9txLiJyi4jkikjuoUOHPBi6Z+wq3cXSLUuZPnA6E/pO8HU4SnXIhAkTGDVqFOnp6SQmJjJq1ChGjRrF7t272/U877zzDl9//XW7f/+ZZ57Jhg0b2n1eo9///vf8+c9/PvmBPvTDH/6QXbt2udxXVFTEOeecQ3h4OPfcc0+bz3HkyBHOO+88Bg8ezEUXXURpaekpx+XNBOHqesa0eBwADAbOAa4ClohIjJvnYox53hiTbYzJTkxMPMVwPcsYw7xV8wgNCOVn2T/zdThKddjq1avZsGEDc+fOZfbs2WzYsIENGzaQkZHRrufpaII4FbW1tbz66qvMnt21S9rcdttt/O53v3O5LywsjPnz5/PYY4+d8Dnmz5/PxRdfzPbt2znrrLN4/PHHTzkub45iKgD6OT1OAwpdHLPKGFMLfCci32AljAKspOF87mdei9QLVuxaQc6BHP5n4v+QEJrg63BUN/bbv28lr7DMo885PCWKX8849Zn8//znP5k7dy7V1dUMHjyYpUuXEh4ezv33388//vEPAgICuPjii5k+fTrvv/8+X3zxBb/5zW949913251gAF577TUee+wxjDFceumlPPLIIwA899xzLFy4kJSUFAYNGkRERARPPvkkK1euZNy4cfj7WyVtVq1axS233EJERASTJ09m5cqVbNiwgZ07d3Lddddx7Ngx/Pz8eOaZZ5gwYQIfffQR8+fPJz4+no0bNzJ79mxOP/10nnrqKaqrq1m+fDkZGRlcffXVREdHk5eXR35+Pn/605948cUXWb16NZMnT+bFF18E4JZbbmHdunUcP36c2bNn86tf/QqAc845h5tuuon6+npHrI0aY922bdsJ35v33nuPVatWAXDttdcydepU5s+f3+732Jk3ryBygMEiMkBEgoArgeUtjnkXOBdARBKwmpx2AR8AF4pIrIjEAhfa27qF0upSfp/7e0YkjmDW6bN8HY5SXlFUVMSCBQv4+OOPWbduHSNGjGDRokUcPHiQ999/n61bt7Jp0yYeeughzjrrLKZNm8YTTzzRoasPsIb+zpkzh08//ZT169fzxRdfsGLFCvbu3cuCBQtYvXo1H374IXl5jnEwfPHFF4wdO9bx+Prrr2fJkiV8+eWXGNPUKNG3b19WrlzJ+vXref311/npT3/q2Ldx40YWL17M5s2bWbJkCbt37yYnJ4drr72Wp59+2nFcaWkpn376KY8//jgzZszggQceIC8vj7Vr17JlyxYAFixYQG5uLhs3bmTlypWOWP39/cnIyHAc1xFHjhyhsSUlNTWV/fv3d/i5GnntCsIYUycid2J9sPsDS40xW0VkLpBrjFlOUyLIA+qB+40xRwBE5GGsJAMw1xhT7K1YPW3RukWUVpfy/AXP4yc6WV2dGk980/eGL7/8kry8PCZNmgRATU0NZ555JnFxcfj5+XHzzTdzySWXMH36dI/8vtWrVzNlyhQSEqwr8h//+Md8/vnnVFVVMWXKFGJjYwGYNWsW+fn5AOzfv5/Ro0cDcPjwYWpqahg/frzj/I8++giA6upq7rzzTjZu3EhAQAA7d+50/N4JEyaQnJwMwMCBA7noImui6xlnnMFXX33lOG7GjBmO7SkpKQwfPhyA4cOHs3v3brKysnjjjTd48cUXqauro7CwkLy8PMdxSUlJFBYWMnLkSI+8X54YBefViXLGmPeB91ts+5XTfQPcZ99anrsUWOrN+LxhQ9EG/vrtX7lm+DUMiRvi63CU8hpjDFOnTuXVV19ttS83N5eVK1eybNkynn32WT788MM2n8f5Q/uKK65wNLu4+n3t2Q4QGhrqGPt/ouMWLlxIv379eO2116itrSUiIsKxLzg42HHfz8/P8djPz4+6urpWxzkf43zc9u3bWbRoEWvWrCEmJoarr7662byEqqoqQkNDeeutt5g3bx4AL730EqNGjWozbmfx8fEcOnSIxMRE9u3bR58+fdw670T0660H1TXU8fCqh0kOS+b2Ubf7OhylvGrSpEn8+9//doy+qaioYPv27ZSXl1NWVsb06dN54oknWL9+PQCRkZGUl5e3ep6goCBHx3dbyQFg4sSJfPrppxw5coS6ujqWLVvG2WefzYQJE/j00085evQotbW1vPPOO45zhg0bxo4dOwBITEwkMDCQ3NxcAJYtW+Y4rrS0lL59+yIivPzyyydMJh1VVlZGZGQkUVFR7N+/nw8+aN5qvn37djIzM5k1a5bj/XA3OQBceumlvPzyywC8/PLLzJzZclZB+2mpDQ96fdvrfFvyLU+e8yThgeG+Dkcpr0pOTubFF19k9uzZ1NTUAPDII48QGhrKFVdcQXV1NQ0NDfzhD38A4KqrruLWW29l4cKFHeqkTktLY+7cuZxzzjkYY5gxYwaXXHIJAPfffz/jx48nNTWVzMxMoqOjAZg2bRo33nij4zmWLl3K9ddfT2RkJN///vcdx915553MmjWLN954g/PPP7/ZFYCnjBkzhuHDh5OVlcXAgQOZPLlpHfrCwkKio6NpazRmWloalZWV1NbW8tZbb/Hxxx8zZMgQrr/+eu6++25GjRrFL37xC370ox/x3HPPMWDAAN58881TD9oY0yNuY8eONb60/9h+M+61ceaOj+4wDQ0NPo1FdX95eXm+DqFbKS8vN8YYU1NTYy6++GKzfPlyx74ZM2aYnTt3NjvOGGPmzZtn7rvvvs4NtA2PP/64eemll7z+e1z9v8LqE3b5uapNTB6yYM0CjDE8NOEhLZGgVCf7n//5H0aPHs2IESMYMmRIs47xxx57jMJCa4T98uXLGTVqFFlZWXz11Vc89NBDvgq5mfj4eK6++mpfh9GKGC+0tflCdna2aWxb7Gyf7f2Muz65i3vG3MONZ9x48hOUOolt27YxbNgwX4ehehhX/69EZK0xJtvV8XoFcYoqayt5dPWjDIoZxDWZ1/g6HKWU8hjtpD5Fz216jsKKQl6a+hKBfu4vxKGUUl2dXkGcgu0l23ll6ytcPuhyxiaPPfkJSinVjWiC6KAG08C8VfOICIrg3rH3+jocpZTyOE0QHfTejvdYV7SO+8beR2xIrK/DUcprtNy397Us952Tk0NWVhaDBg3i3ntdfwE1xnD77bczaNAgRo4ceUrvUVs0QXRASVUJC9cuZEzSGGYOOvXZikp1ZVru2/talvu+7bbb+NOf/sT27dvZunUrK1eubHXO3//+d/bu3cuOHTtYvHgxd9xxh8fj0k7qDvjD2j9QUVPBnIlztBif8r5/PggHNnv2OfucARcvOOWn0XLfni/3XVhYSFVVFePGjQPgv/7rv3j33Xe54IILmr0X7733HtdcY42cPPPMMzlw4ICjFpOnaIJop9wDuby7411uyLqBwbGDfR2OUj7jXO67cVGbRYsWceONNzrKfYsIR48eJSYmhmnTpjFr1iwuu+yyDv2+xnLfubm5REdHc/7557NixQpGjhzJggULWLduHeHh4ZxzzjmO4n+uyn2//PLLjB8/np///OeO7Y3lvkNCQvj666+59tprWb16NWCV+962bRvR0dFkZGRw++23k5OTw8KFC3n66af5/e9/DzSV+3777beZMWMGX331FUOHDmXMmDFs2bKFrKwsFixYQFxcHHV1dZx77rnMmjWL4cOHNyv3ffz4cfr1a1pKJy0tjX379rV6P/bt2+fyOE0QPlJbX8u8VfNICU/h1hG3+joc1Vt44Ju+N2i5b++U+46JiWn12l1VZ3A1ydnTVRy0faQdXs57mZ2lO/nFhF8QFhjm63CU8iljl/tu7JPIy8vj+eefd1RMveyyy3j77bcdBfXaUlNT4+j4njt37gl/X3u2Q/vLfW/evJk1a9ZQXV3t2Ofpct+ffPIJmzZtYurUqS7LfaelpbF3717H9oKCAlJSUlrF7O5xp0IThJsKygt4buNznJd+Hmf3O9vX4Sjlc1ruu33cLffdr18/goODycnJwRjDq6++6rJ096WXXsorr7wCwH/+8x+Sk5M92rwE2sTkFmMMj6x+BBHhwfEP+jocpboELffdPu0p9/3ss89y3XXXUVVVxfTp0x0d1IsXLyY4OJibbrqJGTNm8M9//pPTTjuN8PBwx1oQHtVWmdfudvNmue+Vu1earJeyzEtbvF+OVyljtNx3e2m5b/douW8Pq6it4NE1jzIkdgg/GfYTX4ejlHJBy317h5b7PonHcx7ntbzXeHXaq4xM9Mxi4kqdjJb7Vt6g5b49aNuRbby+7XVmnT5Lk4NSqtfRBNGG+oZ6Hl71MDHBMdw95m5fh6OUUp1OE0Qb3t7+NpsPb+bn2T8nOjja1+EopVSn0wThwuHjh3ly7ZOM7zOe6QM9MwtUKaW6G00QLizMXUhVfRVzJs7x+NR1pbobLfftfS3LfT/44IOkpaW5LLvhbN68eQwaNIihQ4c6yoZ4kk6Ua2H1/tWs2LWCW0fcyoDoAb4ORymfayxa99JLL5Gbm8vTTz/doed555138PPzY+jQoZ4M74Qay32vW7eu035nRzSW+3722WcBmDlzJnfeeSdZWVltnrNp0ybeeecd8vLy2Lt3L1OnTuWbb77Bz89z3/s1QTipqa9h3qp59Ivsx01n3OTrcJQC4LE1j/F1sWfXURgaN5QHxj9wys+j5b49X+7b39+f733ve83qPLny3nvvcdVVVxEUFMRpp51Geno6a9eudZQJ9wRtYnKydMtSdpft5pcTfklIQIivw1GqS3Mu971u3TpGjBjBokWLOHjwoKPc96ZNm3jooYc466yzmDZtGk888USHFhuCpnLfn376KevXr+eLL75gxYoV7N27lwULFrB69Wo+/PBD8vLyHOe4Kve9ZMkSvvzyy2b1lhrLfa9fv57XX3+dn/70p459GzduZPHixWzevJklS5awe/ducnJyuPbaa5tdTTWW+3788ceZMWMGDzzwAHl5eaxdu5YtW7YAsGDBAnJzc9m4cSMrV650xOpc7ttdbZX79iSvXkGIyFRgEeAPLDHGLGix/zrgd0Djq3raGLPE3lcPNK6Skm+MudSbseaX5fPCpheYmjGVyamTT36CUp3EE9/0vUHLfXun3PfIke7NuXI1ydnTfaZeSxAi4g8sBi4ACoAcEVlujMlrceibxpg7XTzFcWPMKG/F58wYw/zV8wnyD+L+cfd3xq9UqtszdrnvV199tdW+3NxcVq5cybJly3j22Wf58MMP23we5w/tK664os2Krm1VfThRNYj2lvt+7bXXqK2tJSIiwrHP0+W+16xZQ0xMDFdffbXLct/u6u7lvscDO4wxu4wxNcAyoEsu4PzB7g/4svBL7hp9F0lhSb4OR6luQct9t4+75b7ddemll/LGG29QU1PDzp072bNnT7PmNE/wZoJIBfY6PS6wt7X0AxHZJCJviUg/p+0hIpIrIqtExOUahSJyi31M7qFDhzoUZHlNOY/lPMbw+OHMHtK1FzZXqitxLvc9cuRIJk2axLfffktpaSmXXHIJI0eOZMqUKc3KfT/yyCMdGiILzct9jxo1iokTJ3LJJZeQnp7uKPd94YUXtir3/e9//9vxHI3lvidNmoSfn1+zct9Llixh4sSJ7Nmzx+vlvm+++eYTlvu+7777yMjIoKysjLS0NObNmwfA3/72N8eiSiNHjuSyyy5j2LBhTJs2jWeeecajI5gA75X7Bn6I1e/Q+Pi/gKdaHBMPBNv3bwM+cdqXYv8cCOwGTjvR7+toue9DlYfMXR/fZbYc2tKh85XyBi333T5a7ts9XancdwHgfEWQBhQ6H2CMOWKMaVzb7wVgrNO+QvvnLuAzYLQ3gkwITeB/p/wvmQnuX9oppboWLfftHV4r9y0iAcC3wHlYo5RygB8bY7Y6HdPXGLPfvn858IAxZqKIxAKVxphqEUkAvgJmmtYd3A7eKvetlC9ouW/lDe0t9+21UUzGmDoRuRP4AGuY61JjzFYRmYt1SbMc+KmIXArUAcXAdfbpw4DnRKQBq59kwYmSg1I9kTFGS70oj+nIxYAuGKRUF/Tdd98RGRlJfHy8Jgl1yowxHDlyhPLycgYMaF5CyCdXEEqpjktLS6OgoICOjs5TqqWQkBDS0tLadY4mCKW6oMDAwFbf9JTqbFqLSSmllEuaIJRSSrmkCUIppZRLPWYUk4gcAvacwlMkAIc9FI4naVzto3G1j8bVPj0xrv7GmERXO3pMgjhVIpLb1lAvX9K42kfjah+Nq316W1zaxKSUUsolTRBKKaVc0gTR5HlfB9AGjat9NK720bjap1fFpX0QSimlXNIrCKWUUi5pglBKKeVSr00QIvJDEdkqIg0i0ubwMBGZKiLfiMgOEXmwE+KKE5GVIrLd/hnbxnH1IrLBvi33YjwnfP0iEiwib9r7V4tIhrdiaUdM14nIIaf35yZvx2T/3qUiUiQiW9rYLyLyv3bcm0RkTBeJ6xwRKXV6v9peGNqzcfUTkU9FZJv9t3i3i2M6/T1zM65Of89EJERE1ojIRjuu37o4xrN/j20tNdfTb1hrTgzBWq0uu41j/IGdWMueBgEbgeFejutx4EH7/oPAY20cd6wT3qOTvn7gduCP9v0rgTe7QEzXAU/74P/U94ExwJY29k8D/gkIMBFY3UXiOgdY4YP3qy8wxr4fibXAWMt/y05/z9yMq9PfM/s9iLDvBwKrgYktjvHo32OvvYIwxmwzxnxzksPGAzuMMbuMMTXAMmCml0ObCbxs338ZuMzLv+9E3Hn9zvG+BZwn3l3AwBf/Jm4xxnyOtfBVW2YCrxjLKiBGRPp2gbh8whiz3xizzr5fDmwDUlsc1unvmZtxdTr7PThmPwy0by1HGXn077HXJgg3pQJ7nR4X4P3/KMnGXobV/pnUxnEhIpIrIqtExFtJxJ3X7zjGGFMHlALxXorH3ZgAfmA3SbwlIv1c7PcFX/x/ctf37KaLf4pIpy/QbjeFjMb6VuzMp+/ZCeICH7xnIuIvIhuAImClMabN98sTf489ej0IEfkI6ONi1y+NMe+58xQutp3yuOATxdWOp0k3xhSKyEDgExHZbIzZeaqxteDO6/fKe3QC7vy+vwNvGGtN89uwvlFN8WJM7urs98pd67Dq8RwTkWnAu8DgzvrlIhIBvA3cY4wpa7nbxSmd8p6dJC6fvGfGmHpglIjEAH8TkSxjjHPfkkffrx6dIIwx55/iUxQAzt8+04DCU3zOE8YlIgdFpK8xZr99KV3UxnMU2j93ichnWN9yPJ0g3Hn9jccUiEgAEI13mzNOGpMx5ojTwxeAx7wYT3t45f/TqXL+8DPGvC8iz4hIgjHG60XpRCQQ60P4dWPMOy4O8cl7drK4fPme2b/zqP13PxVwThAe/XvUJqYTywEGi8gAEQnC6vTx2ogh23LgWvv+tUCrKx0RiRWRYPt+AjAZyPNCLO68fud4ZwGfGLuHzEtOGlOLNupLsdqQu4LlwDX2yJyJQGljc6IviUifxnZqERmP9blw5MRneeT3CvAisM0Y84c2Duv098yduHzxnolIon3lgIiEAucDX7c4zLN/j53ZC9+VbsDlWNm2GjgIfGBvTwHedzpuGtYohp1YTVPejise+BjYbv+Ms7dnA0vs+5OAzVgjeDYDN3oxnlavH5gLXGrfDwH+CuwA1gADO+E9OllMjwJb7ffnU2BoJ/2fegPYD9Ta/7duBG4DbrP3C7DYjnszbYye80Fcdzq9X6uASZ0U15lYzR+bgA32bZqv3zM34+r09wwYAay349oC/Mre7rW/Ry21oZRSyiVtYlJKKeWSJgillFIuaYJQSinlkiYIpZRSLmmCUEop5ZImCKXaQUSOnfyoE57/lj37HRGJEJHnRGSnXZ3zcxGZICJB9v0ePZFVdX2aIJTqJHa9Hn9jzC570xKsWa6DjTGZWFVoE4xVhPBjYLZPAlXKpglCqQ6wZ/b+TkS2iMhmEZltb/ezyy5sFZEVIvK+iMyyT/sJ9sx4ETkNmADMMcY0gFU2xRjzD/vYd+3jlfIZvYRVqmOuAEYBI4EEIEdEPscqe5IBnIFViXcbsNQ+ZzLWrGaATGCDsYqvubIFGOeVyJVyk15BKNUxZ2JVjK03xhwE/o31gX4m8FdjTIMx5gBWqY9GfYFD7jy5nThqRCTSw3Er5TZNEEp1TFuLsJxocZbjWLVywKrjM1JETvQ3GAxUdSA2pTxCE4RSHfM5MNtewCURa1nPNcB/sBYr8hORZKylKRttAwYBGGvtjlzgt05VQQeLyEz7fjxwyBhT21kvSKmWNEEo1TF/w6qquRH4BPh/dpPS21gVU7cAz2GtRFZqn/MPmieMm7AWjtohIpux1q5oXOvgXOB9774EpU5Mq7kq5WEiEmGslcbisa4qJhtjDtg1/D+1H7fVOd34HO8AD5mTr5uulNfoKCalPG+FvbBLEPCwfWWBMea4iPwaa93g/LZOthdCeleTg/I1vYJQSinlkvZBKKWUckkThFJKKZc0QSillHJJE4RSSimXNEEopZRy6f8D7/euRZUHKTMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#accuracy_s1 =np.array(accuracy_s).reshape(len(C_s),len(gamma_s))\n",
    "Otto_SVM_result = pd.read_csv(\"Otto_SVM_result.csv\")\n",
    "accuracy_s1 = Otto_SVM_result['accuracy']\n",
    "\n",
    "C_s = np.logspace(-1, 3, 5)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份 \n",
    "gamma_s = np.logspace(-1, 1, 3)  \n",
    "accuracy_s1 =np.array(accuracy_s1).reshape(len(C_s),len(gamma_s))\n",
    "\n",
    "x_axis = np.log10(C_s)\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    plt.plot(x_axis, np.array(accuracy_s1[:,j]), label = ' Test - log(gamma)' + str(np.log10(gamma)))\n",
    "\n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'accuracy' )\n",
    "plt.savefig('RBF_SVM_Otto.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10.0\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "### 最佳超参数\n",
    "index = np.unravel_index(np.argmax(accuracy_s1, axis=None), accuracy_s1.shape)\n",
    "Best_C = C_s[ index[0] ]\n",
    "Best_gamma = gamma_s[ index[1] ]\n",
    "\n",
    "print(Best_C)\n",
    "print(Best_gamma)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 找到最佳参数后，用全体训练数据训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SVC训练SVC，支持概率输出\n",
    "Best_C = 100\n",
    "Best_gamma = 1.0\n",
    "\n",
    "SVC4 =  SVC( C = Best_C, kernel='rbf', gamma = Best_gamma, probability=True)\n",
    "SVC4.fit(X_train, y_train)\n",
    "\n",
    "#保持模型，用于后续测试\n",
    "import pickle\n",
    "pickle.dump(SVC4, open(\"Otto_RBF_SVC.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
